# coding: utf-8
import requests
from lxml import etree


def collect(url):
    print(url)
    headers = {
        'Referer': 'https://baike.baidu.com/',
        'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.106Safari/537.36'
    }
    res = requests.get(url=url, headers=headers).content.decode('utf-8')
    # open('百度百科.html', 'w', encoding='utf-8').write(res)
    return res


def analyze(level):
    global dic
    if level == 'level0':
        nextLevel = 'level1'
    elif level == 'level1':
        nextLevel = 'level2'
    elif level == 'level2':
        nextLevel = 'level3'
    else:
        nextLevel = ''
    for link in dic[level]:
        url = baseUrl + link
        res = collect(url)
        tree = etree.HTML(res)
        key = tree.xpath('//h1/text()')[0]
        desc = ''.join(tree.xpath('//div[@class="main-content J-content"]//div[@class="para"][1]//text()'))
        sonLs = tree.xpath('//div[@class="main-content J-content"]//div[@class="para"]/a[contains(@href,"/item")]')
        sonKeyLs = []
        for son in sonLs:
            sonLink = son.xpath('./@href')[0]
            sonKey = son.xpath('./text()')[0]
            sonKeyLs.append(sonKey)
            if sonLink not in dic['level0'] + dic['level1'] + dic['level2'] + dic['level3'] and nextLevel:
                dic[nextLevel].append(sonLink)
        sonKeys = '/'.join(sonKeyLs)
        row = f'{clear(key)},{clear(url)},{clear(desc)},{clear(sonKeys)},{clear(level)}'
        print(row)
        open('百度百科.csv', 'a', encoding='utf-8-sig').write(row + '\n')


def clear(string):
    string = string.replace(',', '').replace('\n', '').replace('\r', '')
    return string


if __name__ == '__main__':
    open('百度百科.csv', 'w', encoding='utf-8-sig').write('词条名,url,描述,关键词,层级\n')
    baseUrl = 'https://baike.baidu.com'
    dic = {
        'level0': ['/item/奥拉星/7361319'],
        'level1': [],
        'level2': [],
        'level3': []
    }
    for lev in dic:
        analyze(lev)